package ex2;

import java.io.StringReader;
import java.util.LinkedList;
import java.util.List;

/**
 * Created with IntelliJ IDEA.
 * User: Roi
 * Date: 10/26/13
 */
public class JflexTokenizer implements ITokenizer {
    @Override
    public List<Token> tokenize(String text) throws Exception {
        List<Token> tokens = new LinkedList<Token>();
        Scanner scanner = new Scanner(new StringReader(text));

        Token token = scanner.yylex();
        while (token != null) {
            tokens.add(token);
            token = scanner.yylex();
        }

        return tokens;
    }
}
